{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4d1d9a5c",
   "metadata": {},
   "source": [
    "# Robotics, Vision & Control 3e: for Python\n",
    "## Chapter 12: Image Feature Extraction\n",
    "\n",
    "Copyright (c) 2021- Peter Corke"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a767b56",
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    import google.colab\n",
    "    print('Running on CoLab')\n",
    "    !pip install matplotlib\n",
    "    !pip install machinevision-toolbox-python\n",
    "    COLAB = True\n",
    "except:\n",
    "    COLAB = False\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"last_expr_or_assign\"\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "from math import pi\n",
    "np.set_printoptions(\n",
    "    linewidth=120, formatter={\n",
    "        'float': lambda x: f\"{0:8.4g}\" if abs(x) < 1e-10 else f\"{x:8.4g}\"})\n",
    "np.random.seed(0)\n",
    "from machinevisiontoolbox.base import *\n",
    "from machinevisiontoolbox import *\n",
    "from spatialmath.base import *\n",
    "from spatialmath import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d3147ff",
   "metadata": {},
   "source": [
    "# 12.1 Region Features\n",
    "## 12.1.1 Pixel Classification\n",
    "### 12.1.1.1 Monochrome Image Classification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0d80553",
   "metadata": {},
   "outputs": [],
   "source": [
    "castle = Image.Read(\"castle.png\", dtype=\"float\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8995e1d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "(castle >= 0.7).disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6aef04ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# castle.ithresh()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99152ff0",
   "metadata": {},
   "outputs": [],
   "source": [
    "castle.hist().plot();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de33d951",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = castle.otsu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b57bdfa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "castle2 = Image.Read(\"castle2.png\", dtype=\"float\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dadea83",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = castle2.otsu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69a2953f",
   "metadata": {},
   "outputs": [],
   "source": [
    "castle2.threshold_adaptive(h=15).disp();"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd386d11",
   "metadata": {},
   "source": [
    "### 12.1.1.2 Color Image Classification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "353e12f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = Image.Read(\"yellowtargets.png\", dtype=\"float\", gamma=\"sRGB\");\n",
    "targets.disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78b9f64a",
   "metadata": {},
   "outputs": [],
   "source": [
    "garden = Image.Read(\"tomato_124.png\", dtype=\"float\", gamma=\"sRGB\");\n",
    "garden.disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e074885",
   "metadata": {},
   "outputs": [],
   "source": [
    "ab = targets.colorspace(\"L*a*b*\").plane(\"a*:b*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5f3789c",
   "metadata": {},
   "outputs": [],
   "source": [
    "ab.plane(\"b*:\").disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "583a70ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "targets_labels, targets_centroids, resid = ab.kmeans_color(k=2, seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53dd08bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "targets_labels.disp(colormap=\"jet\", colorbar=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc878867",
   "metadata": {},
   "outputs": [],
   "source": [
    "targets_centroids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a810c4f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "with plt.ioff():\n",
    "    plot_chromaticity_diagram(colorspace=\"a*b*\");\n",
    "    plot_point(targets_centroids, marker=\"*\", text=\"{}\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b945116a",
   "metadata": {},
   "outputs": [],
   "source": [
    "[color2name(c, \"a*b*\") for c in targets_centroids.T]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f382e68b",
   "metadata": {},
   "outputs": [],
   "source": [
    "resid / ab.npixels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "552815bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ab.kmeans_color(centroids=targets_centroids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1aea3bab",
   "metadata": {},
   "outputs": [],
   "source": [
    "objects = (labels == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "edd56e26",
   "metadata": {},
   "outputs": [],
   "source": [
    "objects.disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5a4f6c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "ab = garden.colorspace(\"L*a*b*\").plane(\"a*:b*\")\n",
    "garden_labels, garden_centroids, resid = ab.kmeans_color(k=3, seed=0);\n",
    "garden_centroids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5111d20e",
   "metadata": {},
   "outputs": [],
   "source": [
    "[color2name(c, \"a*b*\") for c in garden_centroids.T]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1733586a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tomatoes = (garden_labels == 2);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7a74edd",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.random.rand(500, 2);  # 500 x 2D data points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc525152",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.cluster.vq import kmeans2\n",
    "centroids, labels = kmeans2(data, k=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02f761db",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(3):\n",
    "  plot_point(data[labels==i, :].T, color=\"rgb\"[i], marker=\".\", markersize=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16c65dea",
   "metadata": {},
   "outputs": [],
   "source": [
    "tomatoes_binary = tomatoes.close(Kernel.Circle(radius=15));\n",
    "tomatoes_binary.disp();"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ac1c82f",
   "metadata": {},
   "source": [
    "### 12.1.1.3 Semantic Classification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e99e37a",
   "metadata": {},
   "outputs": [],
   "source": [
    "scene = Image.Read(\"image3.jpg\")\n",
    "scene.disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fe3dcd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    import torch\n",
    "    import torchvision as tv\n",
    "except ModuleNotFoundError:\n",
    "        print(\"please install PyTorch:  pip install torch torchvision\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f2b952c",
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = tv.transforms.Compose([\n",
    "   tv.transforms.ToTensor(),\n",
    "   tv.transforms.Normalize(mean=[0.485, 0.456, 0.406], \n",
    "                           std=[0.229, 0.224, 0.225])]);\n",
    "in_tensor = transform(scene.image);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47829328",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tv.models.segmentation.fcn_resnet50(pretrained=True).eval();\n",
    "outputs = model(torch.stack([in_tensor]));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32f38fdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = Image(torch.argmax(outputs[\"out\"].squeeze(), dim=0).detach().cpu().numpy());\n",
    "labels.disp(colormap=\"viridis\", ncolors=20, colorbar=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8750ccd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "(labels == 15).disp();\n",
    "scene.choose([255, 255, 255], labels != 15).disp();"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8b94b10",
   "metadata": {},
   "source": [
    "## 12.1.2 Object Instance Representation\n",
    "### 12.1.2.1 Creating Binary Blobs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e04c87f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "sharks = Image.Read(\"sharks.png\");\n",
    "sharks.disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2183ba4",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels, m = sharks.labels_binary()\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a08e2e25",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels.disp(colorbar=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7009930b",
   "metadata": {},
   "outputs": [],
   "source": [
    "right_shark = (labels == 3);\n",
    "right_shark.disp();"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7bd5cbb",
   "metadata": {},
   "source": [
    "### 12.1.2.2 Maximally Stable Extremal Regions (MSER)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1ae9361",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels, m = castle2.labels_MSER()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fb24b9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc99c8cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels.disp(colormap=\"viridis_r\", ncolors=m);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aacc4a54",
   "metadata": {},
   "source": [
    "### 12.1.2.3 Graph-Based Segmentation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "399396f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "grain = Image.Read(\"58060.png\")\n",
    "grain.disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04619abf",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels, m = grain.labels_graphseg()\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "386a672c",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels.disp(colormap=\"viridis_r\", ncolors=m);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4ab4c00",
   "metadata": {},
   "source": [
    "## 12.1.3 Object Instance Description\n",
    "### 12.1.3.1 Area\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b373a53",
   "metadata": {},
   "outputs": [],
   "source": [
    "right_shark.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7333b8c",
   "metadata": {},
   "source": [
    "### 12.1.3.2 Bounding Boxes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b47f40b",
   "metadata": {},
   "outputs": [],
   "source": [
    "u, v = right_shark.nonzero()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94e214c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "u.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f1292f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "umin = u.min()\n",
    "umax = u.max()\n",
    "vmin = v.min()\n",
    "vmax = v.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acea584d",
   "metadata": {},
   "outputs": [],
   "source": [
    "right_shark.disp(block=None);  # display it again, it was a few cells back\n",
    "plot_box(lrbt=[umin, umax, vmin, vmax], color=\"g\");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb27cd54",
   "metadata": {},
   "source": [
    "### 12.1.3.3 Moments\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "393ace0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "m00 = right_shark.mpq(0, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "130bc8e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "uc = right_shark.mpq(1, 0) / m00\n",
    "vc = right_shark.mpq(0, 1) / m00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6ecb362",
   "metadata": {},
   "outputs": [],
   "source": [
    "right_shark.disp(block=None);  # display it again, it was a few cells back\n",
    "plot_point((uc, vc), [\"bo\", \"bx\"]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15840f9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "u20 = right_shark.upq(2, 0); u02 = right_shark.upq(0, 2); u11 = right_shark.upq(1, 1);\n",
    "J = np.array([[u20, u11], [u11, u02]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e08b33db",
   "metadata": {},
   "outputs": [],
   "source": [
    "right_shark.disp(block=None);  # display it again, it was a few cells back\n",
    "plot_ellipse(4 * J  / m00, centre=(uc, vc), inverted=True, color=\"blue\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "158f27e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "lmbda, x = np.linalg.eig(J)\n",
    "lmbda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1923e6b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = 2 * np.sqrt(lmbda.max() / m00)\n",
    "b = 2 * np.sqrt(lmbda.min() / m00)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74335a54",
   "metadata": {},
   "outputs": [],
   "source": [
    "b / a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d053089",
   "metadata": {},
   "outputs": [],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af50d92c",
   "metadata": {},
   "outputs": [],
   "source": [
    "i = np.argmax(lmbda)  # get index of largest eigenvalue\n",
    "v = x[:, i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bfe730f",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.rad2deg(np.arctan2(v[1], v[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e804cd98",
   "metadata": {},
   "source": [
    "### 12.1.3.4 Blob Descriptors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "655686ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs = sharks.blobs();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82314973",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ccb78ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(blobs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d765dc27",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97f9463b",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs[3].area\n",
    "blobs[3].umin\n",
    "blobs[3].aspect\n",
    "blobs[3].centroid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "746e01d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs[3].moments.m00   # moment p=q=0\n",
    "blobs[3].moments.mu11  # central moment p=q=1\n",
    "blobs[3].moments.nu03  # normalized central moment p=0, q=3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "700eee60",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs.area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7048184",
   "metadata": {},
   "outputs": [],
   "source": [
    "sharks.disp(block=None)\n",
    "blobs[3].plot_box(color=\"red\")\n",
    "blobs[:2].plot_box(color=\"red\")\n",
    "blobs.plot_centroid(marker=\"+\", color=\"blue\")\n",
    "blobs.plot_box(color=\"red\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0321413a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sharks.roi(blobs[1].bbox).rotate(blobs[1].orientation).disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "952eba1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs[blobs.area > 10_000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3de2946",
   "metadata": {},
   "outputs": [],
   "source": [
    "tomato_blobs = tomatoes_binary.blobs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e00b453",
   "metadata": {},
   "outputs": [],
   "source": [
    "tomato_blobs.filter(area=(1_000, 5_000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db87a588",
   "metadata": {},
   "outputs": [],
   "source": [
    "tomato_blobs.filter(touch=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b43e0c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "tomato_blobs.filter(area=[1000, 5000], touch=False, color=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1f619b5",
   "metadata": {},
   "source": [
    "### 12.1.3.5 Blob Hieararchy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d29534e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "multiblobs = Image.Read(\"multiblobs.png\");\n",
    "multiblobs.disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0e92872",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels, m = multiblobs.labels_binary()\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5171e88",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs = multiblobs.blobs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8911a0c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs[1].children\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8613f457",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs[1].parent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a21ead9",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs.label_image().disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d708d23",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs.dotfile(show=True);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba58de4e",
   "metadata": {},
   "source": [
    "### 12.1.3.6 Shape from Moments\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8d282c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs = sharks.blobs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41624f5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs.aspect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f78a902",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs.humoments()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32b3cc30",
   "metadata": {},
   "source": [
    "### 12.1.3.7 Shape from Perimeter\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67c6275e",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs[1].perimeter[:, :5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "997ced9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs[1].perimeter.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0497c4d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "sharks.disp(block=None)\n",
    "blobs[1].plot_perimeter(color=\"orange\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d17b999",
   "metadata": {},
   "outputs": [],
   "source": [
    "sharks.disp(block=None);\n",
    "blobs.plot_perimeter(color=\"orange\")\n",
    "blobs.plot_centroid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec37ef34",
   "metadata": {},
   "outputs": [],
   "source": [
    "p = blobs[1].perimeter_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2788035c",
   "metadata": {},
   "outputs": [],
   "source": [
    "blobs.circularity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06c3e244",
   "metadata": {},
   "outputs": [],
   "source": [
    "p = Polygon2(blobs[1].perimeter).moment(0, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb95072f",
   "metadata": {},
   "outputs": [],
   "source": [
    "r, th = blobs[1].polar();\n",
    "plt.plot(r, \"r\", th, \"b\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fca8bd95",
   "metadata": {},
   "outputs": [],
   "source": [
    "for blob in blobs:\n",
    "  r, theta = blob.polar()\n",
    "  plt.plot(r / r.sum());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5508c00",
   "metadata": {},
   "outputs": [],
   "source": [
    "similarity, _ = blobs.polarmatch(1)\n",
    "similarity"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31702576",
   "metadata": {},
   "source": [
    "## 12.1.4 Object Detection using Deep Learning\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "994cf975",
   "metadata": {},
   "outputs": [],
   "source": [
    "scene = Image.Read(\"image3.jpg\")\n",
    "scene.disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb7a3c86",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision as tv\n",
    "transform = tv.transforms.ToTensor();\n",
    "in_tensor = transform(scene.image);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5bfbdfa4",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tv.models.detection.fasterrcnn_resnet50_fpn(pretrained=True).eval();\n",
    "outputs = model(torch.stack([in_tensor]));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f88673bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = outputs[0][\"scores\"].detach().numpy(); # list of confidence scores\n",
    "labels = outputs[0][\"labels\"].detach().numpy(); # list of class names as strings\n",
    "boxes = outputs[0][\"boxes\"].detach().numpy();   # list of boxes as array([x1, y1, x2, y2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d8136c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60c00ca9",
   "metadata": {},
   "outputs": [],
   "source": [
    "scene.disp(block=None);\n",
    "classname_dict = {1: \"person\", 2: \"bicycle\", 3: \"car\", 4: \"motorcycle\", 18: \"dog\"};\n",
    "for score, label, box in zip(scores, labels, boxes):\n",
    "  if score > 0.5:  # only confident detections\n",
    "    plot_labelbox(classname_dict[label], lbrt=box, filled=True, alpha=0.3, \n",
    "                  color=\"yellow\", linewidth=2);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2b9c372",
   "metadata": {},
   "source": [
    "## 12.1.5 Summary\n",
    "# 12.2 Line Features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4186f3a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "points5 = Image.Read(\"5points.png\", dtype=\"float\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cddedc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "square = Image.Squares(number=1, size=256, fg=128).rotate(0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc50287a",
   "metadata": {},
   "outputs": [],
   "source": [
    "edges = square.canny();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "858b55b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "h = edges.Hough();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e29a1bb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "h.plot_accumulator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11f53a6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(h.votes);\n",
    "plt.yscale(\"log\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a77a1b0",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "lines = h.lines(60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36b4c654",
   "metadata": {},
   "outputs": [],
   "source": [
    "church = Image.Read(\"church.png\", mono=True)\n",
    "edges = church.canny()\n",
    "h = edges.Hough();\n",
    "lines = h.lines_p(100, minlinelength=200, maxlinegap=5, seed=0);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01412a9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "church.disp(block=None);\n",
    "h.plot_lines(lines, \"r--\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36322b26",
   "metadata": {},
   "source": [
    "## 12.2.1 Summary\n",
    "# 12.3 Point Features\n",
    "## 12.3.1 Classical Corner Detectors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c59d38f",
   "metadata": {},
   "outputs": [],
   "source": [
    "view1 = Image.Read(\"building2-1.png\", mono=True);\n",
    "view1.disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ca270f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "harris1 = view1.Harris(nfeat=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b5ef650",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(harris1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e17a9272",
   "metadata": {},
   "outputs": [],
   "source": [
    "harris1[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c3225ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "harris1[0].p\n",
    "harris1[0].strength"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "252b0c54",
   "metadata": {},
   "outputs": [],
   "source": [
    "harris1[:5].p\n",
    "harris1[:5].strength"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3decbbb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "view1.disp(block=None, darken=True);\n",
    "harris1.plot();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f12b9422",
   "metadata": {},
   "outputs": [],
   "source": [
    "view1.disp(block=None, darken=True);\n",
    "harris1[::5].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5408b3d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "view1.disp(block=None, darken=True);\n",
    "harris1.subset(20).plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7302e555",
   "metadata": {},
   "outputs": [],
   "source": [
    "harris1 = view1.Harris(nfeat=500, scale=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed6e2598",
   "metadata": {},
   "outputs": [],
   "source": [
    "view1.Harris_corner_strength().disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "179dff8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "view2 = Image.Read(\"building2-2.png\", mono=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2bc1c7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "harris2 = view2.Harris(nfeat=250);\n",
    "view2.disp(block=None, darken=True);\n",
    "harris2.plot();"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea82e023",
   "metadata": {},
   "source": [
    "## 12.3.2 Scale-Space Corner Detectors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd1bd222",
   "metadata": {},
   "outputs": [],
   "source": [
    "foursquares = Image.Read(\"scale-space.png\", dtype=\"float\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54928c42",
   "metadata": {},
   "outputs": [],
   "source": [
    "G, L, s = foursquares.scalespace(60, sigma=2); "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4493c581",
   "metadata": {},
   "outputs": [],
   "source": [
    "L[5].disp(colormap=\"signed\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4556ef7",
   "metadata": {},
   "outputs": [],
   "source": [
    "s[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4a5bbe1",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(s[:-1], [-Ls.image[63, 63] for Ls in L]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13e838c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = findpeaks3d(np.stack([np.abs(Lk.image) for Lk in L], axis=2), npeaks=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02628f9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "foursquares.disp(block=None);\n",
    "for feature in features:\n",
    "  plt.plot(feature[0], feature[1], 'k+')\n",
    "  scale = s[int(feature[2])]\n",
    "  plot_circle(radius=scale * np.sqrt(2), centre=feature[:2], color=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e374494",
   "metadata": {},
   "outputs": [],
   "source": [
    "mona = Image.Read(\"monalisa.png\", dtype=\"float\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e144b391",
   "metadata": {},
   "outputs": [],
   "source": [
    "G, L, _ = mona.scalespace(8, sigma=8);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e47db948",
   "metadata": {},
   "outputs": [],
   "source": [
    "Image.Hstack(G).disp();\n",
    "Image.Hstack(L).disp();"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39a98783",
   "metadata": {},
   "source": [
    "### 12.3.2.1 Scale-Space Point Feature\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d002f54c",
   "metadata": {},
   "outputs": [],
   "source": [
    "sift1 = view1.SIFT(nfeat=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9180b96f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sift1[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f851ab3f",
   "metadata": {},
   "source": [
    "Actually, those SIFT features are very small and associated with the leafs on the trees along the left-hand edge.  Let's select out the bigger and strong SIFT features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58777de9",
   "metadata": {},
   "outputs": [],
   "source": [
    "sift1 = view1.SIFT().filter(percentstrength=50, minscale=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1554b63",
   "metadata": {},
   "outputs": [],
   "source": [
    "view1.disp(block=None, darken=True);\n",
    "sift1.plot(filled=True, color=\"y\", hand=True, alpha=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcaacbc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(sift1.scale, bins=100);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1c1bf52",
   "metadata": {},
   "source": [
    "# 12.4 Applications\n",
    "## 12.4.1 Character Recognition\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a6394fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "if COLAB:\n",
    "    !sudo apt install tesseract-ocr\n",
    "    !pip install pytesseract\n",
    "\n",
    "try:\n",
    "    import pytesseract as tess\n",
    "except ModuleNotFoundError:\n",
    "    print(\"please install pytesseract:\\n * install tesseract binary, see https://tesseract-ocr.github.io/tessdoc/Installation.html\\n * pip install pytesseract\")\n",
    "penguins = Image.Read(\"penguins.png\");\n",
    "ocr = tess.image_to_data(penguins.image < 100, output_type=tess.Output.DICT);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53717b01",
   "metadata": {},
   "outputs": [],
   "source": [
    "for confidence, text in zip(ocr[\"conf\"], ocr[\"text\"]):\n",
    "  if text.strip() != \"\" and float(confidence) > 0:\n",
    "    print(confidence, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02f056ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "penguins.disp(block=None)\n",
    "for i, (text, confidence) in enumerate(zip(ocr[\"text\"], ocr[\"conf\"])):\n",
    "  if text.replace(\" \", \"\") != \"\" and float(confidence) > 50:\n",
    "    plot_labelbox(text,\n",
    "       lb=(ocr[\"left\"][i], ocr[\"top\"][i]), wh=(ocr[\"width\"][i], ocr[\"height\"][i]),\n",
    "       color=\"y\", filled=True, alpha=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72a41cd7",
   "metadata": {},
   "source": [
    "## 12.4.2 Image Retrieval\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b41ff706",
   "metadata": {},
   "outputs": [],
   "source": [
    "images = ImageCollection(\"campus/*.png\", mono=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b656a0fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = [];\n",
    "for image in images:\n",
    "  features += image.SIFT()\n",
    "features.sort(by=\"scale\", inplace=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "261485eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4574184e",
   "metadata": {},
   "outputs": [],
   "source": [
    "features[:10].table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c40f9ce8",
   "metadata": {},
   "outputs": [],
   "source": [
    "supports = [];\n",
    "for feature in features[:400]:\n",
    "   supports.append(feature.support(images))\n",
    "Image.Tile(supports, columns=20).disp(plain=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "504ff95e",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature = features[108]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "826c1d8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "images[feature.id].disp(block=None);\n",
    "feature.plot(filled=True, color=\"y\", hand=True, alpha=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "305e50f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "bag = BagOfWords(features, 2_000, seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52483e13",
   "metadata": {},
   "outputs": [],
   "source": [
    "w = bag.word(108)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d5f3a46",
   "metadata": {},
   "outputs": [],
   "source": [
    "bag.occurrence(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "305cf36d",
   "metadata": {},
   "outputs": [],
   "source": [
    "bag.contains(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b08327a",
   "metadata": {},
   "outputs": [],
   "source": [
    "bag.exemplars(w, images).disp();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d38a3d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "word, freq = bag.wordfreq();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "324ffb4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.max(freq)\n",
    "np.median(freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31bc682c",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar(word, -np.sort(-freq), width=1);  # sort in descending order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2090da3",
   "metadata": {},
   "outputs": [],
   "source": [
    "bag = BagOfWords(features, 2_000, nstopwords=50, seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6100d4cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "v10 = bag.wwfv(10);\n",
    "v10.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c692e76",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim_10 = bag.similarity(v10);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3f6651a",
   "metadata": {},
   "outputs": [],
   "source": [
    "k = np.argsort(-sim_10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b511891e",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = ImageCollection(\"campus/holdout/*.png\", mono=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df2084c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "S = bag.similarity(query);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ddb6324",
   "metadata": {},
   "outputs": [],
   "source": [
    "Image(S).disp(colorbar=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e3c9bd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.argmax(S, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c974f152",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "bag.retrieve(query[0])\n",
    "bag.retrieve(query[1])"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "main_language": "python",
   "notebook_metadata_filter": "-all"
  },
  "kernelspec": {
   "display_name": "dev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "vscode": {
   "interpreter": {
    "hash": "b7d6b0d76025b9176285a6442c3dd6dd39bcfe7241029b7898b7106bd5e9b472"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
